{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from joblib import Parallel, delayed\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_all_data = pd.read_hdf('data/train.h5')\n",
    "test_all_data = pd.read_hdf('data/test.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_all_data[\"type\"] = train_all_data[\"type\"].map({'围网':0,'刺网':1,'拖网':2})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 对速度做归一化\n",
    "train_all_data[\"v\"] = train_all_data['v'] / 360\n",
    "test_all_data[\"v\"] = test_all_data['v'] / 360 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#min-max标准化\n",
    "def minmaxguiyi(df):\n",
    "    return (df - df.min()) / (df.max() - df.min())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 对坐标做归一化\n",
    "train_all_data[\"x\"] = minmaxguiyi(train_all_data[\"x\"])\n",
    "test_all_data[\"x\"] = minmaxguiyi(test_all_data[\"x\"])\n",
    "train_all_data[\"y\"] = minmaxguiyi(train_all_data[\"y\"])\n",
    "test_all_data[\"y\"] = minmaxguiyi(test_all_data[\"y\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def group_feature(df, key, target, aggs):   \n",
    "    agg_dict = {}\n",
    "    for ag in aggs:\n",
    "        agg_dict[f'{target}_{ag}'] = ag\n",
    "    print(agg_dict)\n",
    "    t = df.groupby(key)[target].agg(agg_dict).reset_index()\n",
    "    return t"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_feature(df,train):\n",
    "    tongjilist = ['max','min','mean','sum','count','std','skew'] #'max','min','mean','sum','count','std','skew'\n",
    "    t = group_feature(df, 'ship','x',tongjilist)\n",
    "    train = pd.merge(train, t, on='ship', how='left')\n",
    "    t = group_feature(df, 'ship','y',tongjilist)\n",
    "    train = pd.merge(train, t, on='ship', how='left')\n",
    "    t = group_feature(df, 'ship','v',tongjilist)\n",
    "    train = pd.merge(train, t, on='ship', how='left')\n",
    "    t = group_feature(df, 'ship','d',tongjilist)\n",
    "    train = pd.merge(train, t, on='ship', how='left')\n",
    "    return train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_dt(df):\n",
    "    # 转换时间并做归一化\n",
    "    df['time'] = pd.to_datetime(df['time'], format='%m%d %H:%M:%S')\n",
    "    df['month'] = df['time'].dt.month / 12\n",
    "    df['day'] = df['time'].dt.day / 31\n",
    "    df['hour'] = df['time'].dt.hour / 24\n",
    "    df['minute'] = df['time'].dt.minute / 60\n",
    "    df['second'] = df['time'].dt.second / 60\n",
    "    df['weekday'] = df['time'].dt.weekday / 7\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/carl-hui/.virtualenvs/AI/lib/python3.7/site-packages/ipykernel_launcher.py:1: DeprecationWarning: time.clock has been deprecated in Python 3.3 and will be removed from Python 3.8: use time.perf_counter or time.process_time instead\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'x_max': 'max', 'x_min': 'min', 'x_mean': 'mean', 'x_sum': 'sum', 'x_count': 'count', 'x_std': 'std', 'x_skew': 'skew'}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/carl-hui/.virtualenvs/AI/lib/python3.7/site-packages/ipykernel_launcher.py:6: FutureWarning: using a dict on a Series for aggregation\n",
      "is deprecated and will be removed in a future version. Use                 named aggregation instead.\n",
      "\n",
      "    >>> grouper.agg(name_1=func_1, name_2=func_2)\n",
      "\n",
      "  \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'y_max': 'max', 'y_min': 'min', 'y_mean': 'mean', 'y_sum': 'sum', 'y_count': 'count', 'y_std': 'std', 'y_skew': 'skew'}\n",
      "{'v_max': 'max', 'v_min': 'min', 'v_mean': 'mean', 'v_sum': 'sum', 'v_count': 'count', 'v_std': 'std', 'v_skew': 'skew'}\n",
      "{'d_max': 'max', 'd_min': 'min', 'd_mean': 'mean', 'd_sum': 'sum', 'd_count': 'count', 'd_std': 'std', 'd_skew': 'skew'}\n",
      "{'x_max': 'max', 'x_min': 'min', 'x_mean': 'mean', 'x_sum': 'sum', 'x_count': 'count', 'x_std': 'std', 'x_skew': 'skew'}\n",
      "{'y_max': 'max', 'y_min': 'min', 'y_mean': 'mean', 'y_sum': 'sum', 'y_count': 'count', 'y_std': 'std', 'y_skew': 'skew'}\n",
      "{'v_max': 'max', 'v_min': 'min', 'v_mean': 'mean', 'v_sum': 'sum', 'v_count': 'count', 'v_std': 'std', 'v_skew': 'skew'}\n",
      "{'d_max': 'max', 'd_min': 'min', 'd_mean': 'mean', 'd_sum': 'sum', 'd_count': 'count', 'd_std': 'std', 'd_skew': 'skew'}\n",
      "18.597485\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/carl-hui/.virtualenvs/AI/lib/python3.7/site-packages/ipykernel_launcher.py:10: DeprecationWarning: time.clock has been deprecated in Python 3.3 and will be removed from Python 3.8: use time.perf_counter or time.process_time instead\n",
      "  # Remove the CWD from sys.path while we load stuff.\n"
     ]
    }
   ],
   "source": [
    "start = time.clock()\n",
    "train_all_data = extract_dt(train_all_data)\n",
    "test_all_data = extract_dt(test_all_data)\n",
    "# 做统计特征 将数据压缩成7000和2000\n",
    "train_min_rows = train_all_data.drop_duplicates('ship')\n",
    "test_min_rows = test_all_data.drop_duplicates('ship')\n",
    "\n",
    "train_all_data = extract_feature(train_all_data,train_min_rows)\n",
    "test_all_data = extract_feature(test_all_data,test_min_rows)\n",
    "end = time.clock()\n",
    "print(str(end-start))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 去除不参与训练的列\n",
    "train_all_data = train_all_data.drop(['time',\"ship\",\"x\",\"y\",\"v\",\"d\"], axis=1)\n",
    "test_all_data = test_all_data.drop(['time',\"x\",\"y\",\"v\",\"d\"], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_all_data = test_all_data.sort_values(by=[\"ship\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_all_data.to_hdf('data/train_transform.h5', key='df', mode='w')\n",
    "test_all_data.to_hdf('data/test_transform.h5', key='df', mode='w')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ship</th>\n",
       "      <th>month</th>\n",
       "      <th>day</th>\n",
       "      <th>hour</th>\n",
       "      <th>minute</th>\n",
       "      <th>second</th>\n",
       "      <th>weekday</th>\n",
       "      <th>x_max</th>\n",
       "      <th>x_min</th>\n",
       "      <th>x_mean</th>\n",
       "      <th>...</th>\n",
       "      <th>v_count</th>\n",
       "      <th>v_std</th>\n",
       "      <th>v_skew</th>\n",
       "      <th>d_max</th>\n",
       "      <th>d_min</th>\n",
       "      <th>d_mean</th>\n",
       "      <th>d_sum</th>\n",
       "      <th>d_count</th>\n",
       "      <th>d_std</th>\n",
       "      <th>d_skew</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1740</th>\n",
       "      <td>7000</td>\n",
       "      <td>0.916667</td>\n",
       "      <td>0.096774</td>\n",
       "      <td>0.458333</td>\n",
       "      <td>0.900000</td>\n",
       "      <td>0.533333</td>\n",
       "      <td>0.714286</td>\n",
       "      <td>0.997737</td>\n",
       "      <td>0.974054</td>\n",
       "      <td>0.985041</td>\n",
       "      <td>...</td>\n",
       "      <td>373</td>\n",
       "      <td>0.008033</td>\n",
       "      <td>1.752080</td>\n",
       "      <td>360</td>\n",
       "      <td>0</td>\n",
       "      <td>137.356568</td>\n",
       "      <td>51234</td>\n",
       "      <td>373</td>\n",
       "      <td>115.674525</td>\n",
       "      <td>0.400544</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>636</th>\n",
       "      <td>7001</td>\n",
       "      <td>0.916667</td>\n",
       "      <td>0.419355</td>\n",
       "      <td>0.958333</td>\n",
       "      <td>0.983333</td>\n",
       "      <td>0.133333</td>\n",
       "      <td>0.142857</td>\n",
       "      <td>0.585858</td>\n",
       "      <td>0.567477</td>\n",
       "      <td>0.578244</td>\n",
       "      <td>...</td>\n",
       "      <td>458</td>\n",
       "      <td>0.006427</td>\n",
       "      <td>1.127644</td>\n",
       "      <td>356</td>\n",
       "      <td>0</td>\n",
       "      <td>149.606987</td>\n",
       "      <td>68520</td>\n",
       "      <td>458</td>\n",
       "      <td>104.616537</td>\n",
       "      <td>0.022398</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1398</th>\n",
       "      <td>7002</td>\n",
       "      <td>0.916667</td>\n",
       "      <td>0.645161</td>\n",
       "      <td>0.958333</td>\n",
       "      <td>0.916667</td>\n",
       "      <td>0.916667</td>\n",
       "      <td>0.142857</td>\n",
       "      <td>0.819748</td>\n",
       "      <td>0.734099</td>\n",
       "      <td>0.787716</td>\n",
       "      <td>...</td>\n",
       "      <td>410</td>\n",
       "      <td>0.007131</td>\n",
       "      <td>0.835796</td>\n",
       "      <td>359</td>\n",
       "      <td>0</td>\n",
       "      <td>159.436585</td>\n",
       "      <td>65369</td>\n",
       "      <td>410</td>\n",
       "      <td>113.566486</td>\n",
       "      <td>0.168657</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>342</th>\n",
       "      <td>7003</td>\n",
       "      <td>0.916667</td>\n",
       "      <td>0.548387</td>\n",
       "      <td>0.458333</td>\n",
       "      <td>0.983333</td>\n",
       "      <td>0.316667</td>\n",
       "      <td>0.714286</td>\n",
       "      <td>0.562346</td>\n",
       "      <td>0.536136</td>\n",
       "      <td>0.540803</td>\n",
       "      <td>...</td>\n",
       "      <td>425</td>\n",
       "      <td>0.007459</td>\n",
       "      <td>2.576536</td>\n",
       "      <td>360</td>\n",
       "      <td>0</td>\n",
       "      <td>122.242353</td>\n",
       "      <td>51953</td>\n",
       "      <td>425</td>\n",
       "      <td>119.624570</td>\n",
       "      <td>0.550111</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>117</th>\n",
       "      <td>7004</td>\n",
       "      <td>0.916667</td>\n",
       "      <td>0.322581</td>\n",
       "      <td>0.458333</td>\n",
       "      <td>0.683333</td>\n",
       "      <td>0.266667</td>\n",
       "      <td>0.714286</td>\n",
       "      <td>0.658395</td>\n",
       "      <td>0.630600</td>\n",
       "      <td>0.634737</td>\n",
       "      <td>...</td>\n",
       "      <td>398</td>\n",
       "      <td>0.007546</td>\n",
       "      <td>2.136030</td>\n",
       "      <td>355</td>\n",
       "      <td>0</td>\n",
       "      <td>123.839196</td>\n",
       "      <td>49288</td>\n",
       "      <td>398</td>\n",
       "      <td>116.631778</td>\n",
       "      <td>0.476934</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 35 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      ship     month       day      hour    minute    second   weekday  \\\n",
       "1740  7000  0.916667  0.096774  0.458333  0.900000  0.533333  0.714286   \n",
       "636   7001  0.916667  0.419355  0.958333  0.983333  0.133333  0.142857   \n",
       "1398  7002  0.916667  0.645161  0.958333  0.916667  0.916667  0.142857   \n",
       "342   7003  0.916667  0.548387  0.458333  0.983333  0.316667  0.714286   \n",
       "117   7004  0.916667  0.322581  0.458333  0.683333  0.266667  0.714286   \n",
       "\n",
       "         x_max     x_min    x_mean  ...  v_count     v_std    v_skew  d_max  \\\n",
       "1740  0.997737  0.974054  0.985041  ...      373  0.008033  1.752080    360   \n",
       "636   0.585858  0.567477  0.578244  ...      458  0.006427  1.127644    356   \n",
       "1398  0.819748  0.734099  0.787716  ...      410  0.007131  0.835796    359   \n",
       "342   0.562346  0.536136  0.540803  ...      425  0.007459  2.576536    360   \n",
       "117   0.658395  0.630600  0.634737  ...      398  0.007546  2.136030    355   \n",
       "\n",
       "      d_min      d_mean  d_sum  d_count       d_std    d_skew  \n",
       "1740      0  137.356568  51234      373  115.674525  0.400544  \n",
       "636       0  149.606987  68520      458  104.616537  0.022398  \n",
       "1398      0  159.436585  65369      410  113.566486  0.168657  \n",
       "342       0  122.242353  51953      425  119.624570  0.550111  \n",
       "117       0  123.839196  49288      398  116.631778  0.476934  \n",
       "\n",
       "[5 rows x 35 columns]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_all_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
